# Scrapy settings for crawler project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'crawler'

SPIDER_MODULES = ['crawler.spiders']
NEWSPIDER_MODULE = 'crawler.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'crawler (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 6

# Retry 的请求优先级(最高)
RETRY_PRIORITY_ADJUST = 0
# Retry 次数
RETRY_TIMES = 10  #15  # initial response + 2 retries = 3 requests

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 0.2    # 当有代理时,这个表示请求最低间隔, 否则它应该是原本的意思
# 无代理时请求间隔(使用请求延迟中间件)
DOWNLOAD_DELAY_DEFAULT = 2.5
# The download delay setting will honor only one of:
CONCURRENT_REQUESTS = 16
CONCURRENT_REQUESTS_PER_DOMAIN = 8

DOWNLOAD_TIMEOUT = 30  # 60秒
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
  'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24',
}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'crawler.middlewares.CrawlerSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html2SWR
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
    # 'crawler.middlewares.ExceptionMiddleware': 600,
    'crawler.middlewares.DelayMiddleware': 800,
    'crawler.middlewares.ProxyMiddleware': 123,
    'crawler.middlewares.CustomRetryMiddleware': 443,
    # 'scrapy_ajax_utils.selenium.SeleniumDownloaderMiddleware': 600

}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    # 'crawler.pipelines.CrawlerPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

# Logging
import datetime
import logging
LOG_LEVEL = 'INFO'
_to_day = datetime.datetime.now()
from os import path
_log_file_path = '{}/log/scrapy_{}_{}_{}.log'.format(path.dirname(__file__),_to_day.year, _to_day.month, _to_day.day)
LOG_FILE = _log_file_path

# 控制台
_console_handler = logging.StreamHandler()
_formatter = logging.Formatter(
            '[%(asctime)s] - %(levelname)s: %(message)s')
_console_handler.setFormatter(_formatter)
_console_handler.setLevel(logging.INFO)
logging.getLogger().addHandler(_console_handler)
# logging.getLogger().addHandler(logging.StreamHandler())


# selenium
from shutil import which
# Default: chrome
# Only support chrome & firefox.
SELENIUM_DRIVER_NAME = 'chrome'
# Default: None
SELENIUM_DRIVER_PATH = which('chromedriver')
# Default: True
SELENIUM_HEADLESS = True
# Default: 30
SELENIUM_DRIVER_PAGE_LOAD_TIMEOUT = 12

# Set the min/max concurrent drivers to download page.
# Default: 3
# SELENIUM_MIN_DRIVERS = 5

# Default: 5
# SELENIUM_MAX_DRIVERS = 10